# -*- coding: utf-8 -*-
import re
from scrapys.items import Www89ipItem, ScrapysItem, ChinadrugtrialsItem
from scrapy.conf import settings
import pymongo

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


class ScrapysPipeline(object):
    def __init__(self):
        self.rep = re.compile(r'\n|&nbsp|\xa0|\\xa0|\u3000|\\u3000|\\u0020|\u0020|\t|\r|:')
        host = settings['MONGODB']['MONGODB_HOST']
        port = settings['MONGODB']['MONGODB_PORT']
        dbname = settings['MONGODB']['MONGODB_DBNAME']
        client = pymongo.MongoClient(host, port)
        self.mdb = client[dbname]
        pass

    def process_item(self, item, spider):
        if isinstance(item, ScrapysItem):
            data = ''
            with open(self.settings['DIR_IPS']['kuaidaili'], 'a+', encoding='utf-8') as f:
                ip = [self.rep.sub('', i) for i in item['ip']]
                port = [self.rep.sub('', i) for i in item['port']]
                type = [self.rep.sub('', i) for i in item['type']]

                for i, j in zip(ip, port):
                    data += i + ':'+ j + "\n"
                f.write(data)
                f.close()
            return item
        elif isinstance(item, Www89ipItem):
            data = ''
            with open(self.settings['DIR_IPS']['www89ip'], 'a+', encoding='utf-8') as f:
                ip = [self.rep.sub('', i) for i in item['ip']]
                port = [self.rep.sub('', i) for i in item['port']]
                type = [self.rep.sub('', i) for i in item['type']]

                for i, j in zip(ip, port):
                    data += i + ':'+ j + "\n"
                f.write(data)
                f.close()
            return item
            pass
        elif isinstance(item, ChinadrugtrialsItem):
            db = self.mdb['test1']

            db.insert(dict(item))
            return item
